<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  <meta name="description"
        content="Editing Conceptual Knowledge for Large Language Models">
  <meta name="keywords" content="ConceptEdit, Model Editing, Conceptual Knowledge">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>Editing Conceptual Knowledge for Large Language Models</title>

  <!-- Global site tag (gtag.js) - Google Analytics -->
  <script async src="https://www.googletagmanager.com/gtag/js?id=G-PYVRSFMDRL"></script>
  <script>
    window.dataLayer = window.dataLayer || [];

    function gtag() {
      dataLayer.push(arguments);
    }

    gtag('js', new Date());

    gtag('config', 'G-PYVRSFMDRL');
  </script>

  <link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro"
        rel="stylesheet">

  <link rel="stylesheet" href="./static/css/bulma.min.css">
  <link rel="stylesheet" href="./static/css/bulma-carousel.min.css">
  <link rel="stylesheet" href="./static/css/bulma-slider.min.css">
  <link rel="stylesheet" href="./static/css/fontawesome.all.min.css">
  <link rel="stylesheet"
        href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css">
  <link rel="icon" href="./static/images/meta.png">
  <link rel="stylesheet" href="./static/css/index.css">

  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
  <script defer src="./static/js/fontawesome.all.min.js"></script>
  <script src="./static/js/bulma-carousel.min.js"></script>
  <script src="./static/js/bulma-slider.min.js"></script>
  <script src="./static/js/index.js"></script>

  <style>
		/* Define the grid layout */
		.mygrid {
			display: grid;
			grid-template-columns: repeat(3, 1fr);
			grid-gap: 20px;
			width: 80%;
			margin: auto;
		}
		.grid_item {
      background: #FFFFFF;
      opacity: 1;
    }

		/* Define the size of the GIFs */
		.mygif {
			height: auto;
			cursor: pointer;
		}
		
		/* Define the modal styles */
		.modal {
			display: none;
			position: fixed;
			z-index: 1;
			left: 0;
			top: 0;
			width: 100%;
			height: 100%;
			overflow: auto;
			background-color: rgba(0,0,0,0.9);
		}
		
		.modal-content {
			margin: auto;
			display: block;
			width: 80%;
			max-width: 800px;
			max-height: 80%;
		}

    /* Define the full-screen overlay styles */
		.overlay {
			position: fixed;
			z-index: 999;
			left: 0;
			top: 0;
			width: 100%;
			height: 100%;
			overflow: hidden;
			background-color: rgba(0,0,0,0.9);
			display: none;
		}
		
		.overlay img {
			width: auto;
			height: 90%;
			margin: 0 auto;
			display: block;
			max-width: 90%;
			max-height: 90%;
		}

    /* Define the video styles */
		.gifvideo {
			width: 100%;
			height: auto;
		}

		/* Define the progress bar styles */
		.progress {
			width: 100%;
			height: 10px;
			background-color: #ddd;
			position: relative;
		}

		.progress-bar {
			height: 100%;
			background-color: #4CAF50;
			position: absolute;
			top: 0;
			left: 0;
		}
		
		/* Define the close button style */
		.close {
			color: white;
			position: absolute;
			top: 10px;
			right: 25px;
			font-size: 35px;
			font-weight: bold;
			cursor: pointer;
		}
		
		.close:hover,
		.close:focus {
			color: #bbb;
			text-decoration: none;
			cursor: pointer;
		}
	</style>
  </head>
  <body>


<section class="hero">
  <div class="hero-body">
    <div class="container is-max-desktop">
      <div class="columns is-centered">
        <div class="column has-text-centered">
          <h2 class="title is-2 publication-title" style="width: 110%; margin-left: -5%">Editing Conceptual Knowledge for Large Language Models</h2>
          <div class="is-size-5">
            <span class="author-block" style="color:#00A4EF;font-weight:normal;">
              Xiaohan Wang<sup>&#x2660;&#x2661;</sup>
            </span>, 
            <span class="author-block" style="color:#00A4EF;font-weight:normal;">
              Shengyu Mao<sup>&#x2660;&#x2661;</sup>
            </span>, 
            <span class="author-block" style="color:#00A4EF;font-weight:normal;">
              Ningyu Zhang<sup>&#x2660;&#x2661;*</sup>
            </span>, 
            <span class="author-block" style="color:#00A4EF;font-weight:normal;">
              Shumin Deng<sup>&#x2663;</sup>
            </span>,
            <span class="author-block" style="color:#00A4EF;font-weight:normal;">
              Yunzhi Yao<sup>&#x2660;&#x2661;</sup>
            </span>,
            <span class="author-block" style="color:#00A4EF;font-weight:normal;">
              Yue Shen<sup>&#x2662;</sup>
            </span>,
            <span class="author-block" style="color:#00A4EF;font-weight:normal;">
              Lei Liang<sup>&#x2662;</sup>
            </span>,
            <span class="author-block" style="color:#00A4EF;font-weight:normal;">
              Jinjie Gu<sup>&#x2662;</sup>
            </span>,
            <span class="author-block" style="color:#00A4EF;font-weight:normal;">
              Huajun Chen<sup>&#x2660;&#x2661;*</sup>
            </span>,
          </div>

          <br>
          <div class="is-size-5 publication-authors">
            <span class="author-block">
              <sup>&#x2660;</sup>Zhejiang University
            </span>
            <span class="author-block">
              <sup>&#x2661;</sup>ZJU-Ant Group Joint Research Center for Knowledge Graphs
            </span>
            <span class="author-block">
              <sup>&#x2663;</sup>National University of Singapore
            </span>
            <span class="author-block">
              <sup>&#x2662;</sup>Ant Group
            </span>
          </div>

          <div class="is-size-5 publication-authors">
            <span class="author-block"><sup>*</sup>Corresponding Author</span>
           
          </div>

          <div class="column has-text-centered">
            <div class="publication-links">
              <!-- PDF Link. -->
              <span class="link-block">
                <a href="https://arxiv.org/abs/2403.06259" target="_blank" 
                   class="external-link button is-normal is-rounded">
                  <span class="icon">
                      <i class="ai ai-arxiv"></i>
                  </span>
                  <span>ArXiv</span>
                </a>
              </span>
              <!-- HF Paper. -->
              <span class="link-block">
                <a href="https://huggingface.co/datasets/zjunlp/ConceptEdit" target="_blank" 
                   class="external-link button is-normal is-rounded">
                  <span class="icon">
                    <p style="font-size:18px">🤗</p>
                  </span>
                  <span>HF Dataset</span>
                </a>
              </span>
              <!-- Code Link. -->
              <span class="link-block">
                <a href="https://github.com/zjunlp/EasyEdit" target="_blank" 
                   class="external-link button is-normal is-rounded">
                  <span class="icon">
                      <i class="fab fa-github"></i>
                  </span>
                  <span>Code</span>
                  </a>
              </span>
              <!-- Google Drive Link. -->

              <span class="link-block">
                <a href="https://drive.google.com/drive/folders/1Hp1DfIuj6Ih6ZLVENS-UmgJT8mRBlFC2?usp=drive_link" target="_blank" 
                  class="external-link button is-normal is-rounded">
                  <span class="icon">
                      <img src="./static/images/drive.png" alt="Drive" style="height: 17px; width: 17px; vertical-align: middle;"/>
                  </span>
                <span>Google Drive</span>
                </a>
              </span>
            </div>

          </div>
        </div>
      </div>
    </div>
  </div>
</section>

<!-- <section class="hero teaser">
  <div class="container is-max-desktop">
    <div class="hero-body">
      <img id="teaser" width="100%" src="./images/first.gif">

      <h2 class="subtitle has-text-centered">
        Armed with just one tool library, the <b>Meta-Agent</b> can automatically differentiate based on the target task information and produce a sub-agent group that can collaborate to complete the task.
      </h2>
    </div>
  </div>
</section> -->

<section class="section">
  <div class="container is-max-desktop">
    <!-- Abstract. -->
    <div class="columns is-centered has-text-centered">
      <div class="column is-four-fifths">
        <h2 class="title is-3">Abstract</h2>
        <div class="content has-text-justified">
          <p>
            Recently, there has been a growing interest in knowledge editing for Large Language Models (LLMs). Current approaches and evaluations merely explore the instance-level editing, while whether LLMs possess the capability to modify concepts remains unclear. This paper pioneers the investigation of editing conceptual knowledge for LLMs, by constructing a novel benchmark dataset <strong>ConceptEdit</strong> and establishing a suite of new metrics for evaluation. The experimental results reveal that, although existing editing methods can efficiently modify concept-level definition to some extent, they also have the potential to distort the related instantial knowledge in LLMs, leading to poor performance. We anticipate this can inspire further progress in better understanding LLMs.
          </p>
        </div>
      </div>
    </div>
    <!--/ Abstract. -->
    <br>
    <br>
    <!-- Introduction. -->
    <div class="columns is-centered has-text-centered">
      <div class="column is-six-fifths">
        <h2 class="title is-3">Conceptual Knowledge Editing</h2>
        <img id="model" width="70%" height="70%" src="./static/images/flow1.gif">
        <!-- <p class="has-text-centered">
          Figure 1: <b>Conceptual Knowledge Editing</b>.
        </p> -->
        <br>
        <div class="column has-text-justified">
          The distinctiveness of human cognition leads to research question: <strong>whether LLMs learn and update concepts analogously</strong>, as well as the manner in which these models encapsulate and retain concepts within parametric framework.

          To this end, we propose <strong>ConceptEdit</strong> (CC BY-NC-SA 4.0 license), a novel benchmark dataset for editing conceptual knowledge, which tries to modify the definition of concepts in LLMs.
          It is constructed upon the foundation of DBpedia Ontology, a widely recognized and cross-domain ontology that preserves conceptual knowledge hierarchically.
          When developing ConceptEdit, we meticulously supplement concepts with their corresponding definitions and associated instances, accompanied by rigorous processions to guarantee its quality. 
        </div>
      </div>
    </div>
    <br>
    <br>
    <!-- Paper Model. -->
    <!-- Paper Model. -->
    <div class="columns is-centered has-text-centered">
      <div class="column is-six-fifths">
        <h2 class="title is-3">ConceptEdit</h2>
        <img id="model" width="100%" src="./static/images/flow.gif">
        <p class="has-text-centered">
          <b>Overview of ConceptEdit benchmark construction</b>.
        </p>
        <br>
        <div class="column has-text-justified">
          <p>Building on the DBpedia Ontology, we enrich concepts with detailed definitions and associated instances, ensuring quality through the following six processes and manually review all the descriptions we gathered:</p>
          <ul>
            <li><strong>&bull; Concept Selection:</strong> DBpedia ontology, a tree-like structure, to assemble a collection of concepts.</li>
            <li><strong>&bull; Concept Completion</strong>: SPARQL to interrogate 20 instances and Wikidata to augment it with descriptive content.</li>
            <li><strong>&bull; Descriptor Generation</strong>: a manually curated template and a distinct concept chosen to supplant the original definition.</li>
            <li><strong>&bull; Neighbour Construction</strong>: restructured examples as equivalent and out-scope from the remaining pool.</li>
            <li><strong>&bull; Instance Filtration</strong>: meticulously sieved the instances to ascertain that LLMs possess a priory knowledge.</li>
            <li><strong>&bull; Intra vs. Inter Split</strong>: testing editing in same superclass Intra and different superclass Inter categories.</li>
          </ul>
        </div>
        
      </div>
    </div>
    <br>
    <br>
    <!-- Paper Model. -->
    
    <!-- Paper Main Results -->
    <div class="columns is-centered has-text-centered">
      <div class="column is-six-fifths">
        <h2 class="title is-3">Main Results</h2>
        <img id="model" width="80%" src="static/images/main_result.png">
        <p class="has-text-centered">
            Table 1: Main results of the baselines on the ConceptEdit. <strong>Bold</strong>  results denote the best performance in each setting, 
            while <u>underlined</u> results signify the second-best. &uarr; means the metric goes higher if it performs better. <b>Gen.</b> is the abbreviation of metric Generalization and <b>Inst. </b>is the abbreviation of metric Instance Change.
          </p>
        <br>
            
        <br>
        <img id="model" width="80%" src="static/images/concept_consistency.png">
        <p class="has-text-centered">
          Figure 3: The results of the <strong>Concept Consistency</strong> employed on the <strong>LLaMA-2-7B-Chat</strong> across both intra and inter modules. 
          This investigation entailed a comparison of generated sentences both pre-edited and post-edited via different editing methods.
        </p>
        <p class="has-text-centered">
          The evidence clearly indicates that FT surpasses other methodologies.
        </p>
        <br>
      </div>
    </div>
    <br>
    <br>
    <!-- Paper Main Results -->

    <!-- Paper Analysis -->
    <div class="columns is-centered has-text-centered">
      <div class="column is-six-fifths">
        <h2 class="title is-3">Analysis</h2>
        <br>

        <p class="has-text-centered">  
          &#x2665;  The gap between Reliability and Concept Consistency signals the necessity for concept specific evaluation metrics.
        </p>
        <img id="model" width="60%" src="./static/images/case3.png">
        <p class="has-text-centered">
          Figure 4: Cases of Reliability Scores vs Generated Sentences. This Figure lists four representative cases that showcase the discrepancy.
        </p>

        <br>
        <p class="has-text-centered">
          &#x2665;  The impact of concepts' structure on editing effects across superclasses but NOT hierarchy.
        </p>

        <img id="model" width="60%" src="./static/images/Figure_1.png">
        <p class="has-text-centered">
         Figure 5: Considering concepts as tree-like structure, we assess the successful edits on mid-hierarchy and leaf-node concepts. 
        </p>
        
        <br>
        <p>&#x2665;  Locating conceptual knowledge emphases the attention mechanism within early few layers.</p>
        <img id="model" width="60%" src="./static/images/ct_case_publisher.png">
        <p class="has-text-centered">
          FIgure 6: To further explore the storage patterns and mechanisms of correlation between concepts and instances, we use casual tracing, identifying neuron activations in LLaMA-2-7B-Chat which has 32 transformer layers. The conceptual and instantial knowledge locating for the concept <strong>publisher</strong> and its corresponding instances by perturbing the input tokens.
        </p>
        
        <br>
        <p>
          &#x2665;   Generated sentence shows varying degrees of success in edits.
        </p>
        <img id="model" width="80%" src="static/images/case.png">
        <p class="has-text-centered">
         Figure 7: Diverse scenarios showcasing the model’s range of outcomes, from successful editing executions to cases of failure. <strong>[TARGET]</strong> denotes the revised description. <strong>[ORIGIN]</strong> refers to the initial recognition prior to editing.
        </p>
      </div>
    </div>
    <!-- Paper Analysis. -->
    <br>
    <br>
    <!-- Paper Acknowledgement. -->
    <div class="columns is-centered has-text-centered">
      <div class="column is-six-fifths">
        <h2 class="title is-3">Acknowledgement</h2>
        <br>

        <p class="has-text-centered">  
          We would like to express our sincere gratitude to <a href="https://www.dbpedia.org/resources/ontology/">DBpedia</a>，<a href="https://www.wikidata.org/wiki/Wikidata:Introduction">Wikidata</a>，<a href="https://github.com/vickywu1022/OntoProbe-PLMs">OntoProbe-PLMs</a> and <a href="https://github.com/kmeng01/rome">ROME</a>.
        </p>
        <br>
        <p>
          Their contributions are invaluable to the advancement of our work.
        </p>
      </div>
    </div>
    <!-- Paper Acknowledgement. -->
  </div>
</section>


<section class="section" id="BibTeX">
  <div class="container is-max-desktop content">
    <h2 class="title">BibTeX</h2>
    <pre><code>
	@misc{wang2024editing,
	      title={Editing Conceptual Knowledge for Large Language Models}, 
	      author={Xiaohan Wang and Shengyu Mao and Ningyu Zhang and Shumin Deng and Yunzhi Yao and Yue Shen and Lei Liang and Jinjie Gu and Huajun Chen},
	      year={2024},
	      eprint={2403.06259},
	      archivePrefix={arXiv},
	      primaryClass={cs.CL}
	}
</code></pre>
  </div>
</section>

<section class="section" id="Acknowledgement">
  <div class="container is-max-desktop content">
    <p>
      This website is adapted from <a
      href="https://github.com/nerfies/nerfies.github.io">Nerfies</a>, licensed under a <a rel="license"
                                          href="http://creativecommons.org/licenses/by-sa/4.0/">Creative
      Commons Attribution-ShareAlike 4.0 International License</a>.
    </p>
  </div>
</section>


<script>
  $(".grid_item").hover(function () {
    $(this).css("background", "#f2f1f1");
    }, 
    function () {
        $(this).css("background", "#FFFFFF"); 
    });

  // Get the modal element
  // var modal = document.getElementById("myModal");
  var overlay = document.getElementById("overlay");
  var span = document.getElementsByClassName("close")[0];


  // Get the image element and the close button element
  //  // display the GIF as it is
  // var img = document.getElementById("modalImg");
  // var img = document.getElementById("overlayImg");
  // Add event listeners to each GIF element
  var gifs = document.getElementsByClassName("mygif");
  for (var i = 0; i < gifs.length; i++) {
  gifs[i].addEventListener("click", function() {
      //  // display the GIF as it is
      // // Set the modal image source and display the modal
      // img.src = this.src;

      // display the GIF as a new image, will play from the begining
      var img = document.createElement("img");
      img.src = this.src.replace(".png", ".gif");

      // Add the img element to the overlay content and display the overlay
      document.getElementById("overlayContent").appendChild(img);
      

      // modal.style.display = "block";
      overlay.style.display = "block";

      // Hide the body overflow
              document.body.style.overflow = "hidden";
  });
  }

  // Add event listener to close button
  span.addEventListener("click", function() {
  // Remove the img element from the overlay content, hide the overlay, and restore the body overflow
          document.getElementById("overlayContent").innerHTML = "";

  // Hide the modal
  // modal.style.display = "none";
  overlay.style.display = "none";
  document.body.style.overflow = "auto";
  });
</script>
</body>
</html>
